package cn.sccdlg.crawler.job.task;

import cn.sccdlg.crawler.job.pojo.JobInfo;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * @author RaoShuang
 * @emil 991207823@qq.com
 * @date 2020/11/30 22:48
 * 51job招聘信息抓取任务执行器
 */
@Component
public class JobProcessor implements PageProcessor {

    /** 51job检索java，计算机+互联网电商 */
    private static final String URL_51JOB = "https://search.51job.com/list/000000,000000,0000,01%252c32,9,99,java,2,1.html" +
            "?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0" +
            "&line=&welfare=";
    private int page = 2;

    @Override
    public void process(Page page) {
        // 解析页面，获取javascript中的招聘信息
        List<Selectable> nodes = page.getHtml().xpath("//script[@type=text/javascript]")
                .regex("\"job_href\":\"(.*?)\"").nodes();
        //判断获取到集合是否为空
        if (nodes.size() == 0) {
            //如果为空，表示这是招聘详情页
            this.saveJobInfo(page);
        } else {
            //如果不为空，表示这是列表页，解析出详情页url地址，放入任务队列中
            for (Selectable selectable : nodes) {
                //获取url地址，并去除多余的转义符
                String jobInfoUrl = selectable.toString();
                jobInfoUrl = jobInfoUrl.replace("\\", "");
                System.out.println(jobInfoUrl);
                //把获取到的url地址放到任务队列中
                page.addTargetRequest(jobInfoUrl);
            }
            if (this.page < 100) {
                //下一页的url
                String nextUrl = "https://search.51job.com/list/000000,000000,0000,00,9,99,java,2," + this.page
                        + ".html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&"
                        + "companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
                //把url放到任务队列中
                page.addTargetRequest(nextUrl);
                this.page = this.page + 1;
            }
        }
    }

    /**
     * 解析页面，获取招聘详情信息，保存数据
     *
     * @param page
     */
    private void saveJobInfo(Page page) {
        //创建招聘详情对象
        JobInfo jobInfo = new JobInfo();
        //解析页面
        Html html = page.getHtml();
        //获取数据，封装到对象中
        jobInfo.setCompanyName(html.css("p.cname > a", "text").toString());
        String tmp = html.css("p.ltype", "title").toString();
        if (tmp != null) {
            String[] strings = tmp.split("\\|");
            jobInfo.setCompanyAddr(strings[0]);
        } else {
            jobInfo.setCompanyAddr("-");
        }
        jobInfo.setCompanyInfo(Jsoup.parse(html.css("div.tmsg").toString()).text());
        jobInfo.setJobName(html.css("div.in > div.cn > h1", "text").toString());
        jobInfo.setJobAddr(html.css("div.bmsg > p.fp", "text").toString());
        jobInfo.setJobInfo(Jsoup.parse(html.css("div.job_msg").toString()).text());
        jobInfo.setUrl(page.getUrl().toString());
        //获取薪资
        Integer[] salary = MathSalary.getSalary(html.css("div.cn strong", "text").toString());
        jobInfo.setSalaryMin(salary[0]);
        jobInfo.setSalaryMax(salary[1]);
        //获取发布时间
        String time = html.css("p.ltype", "title").regex("\\d\\d-\\d\\d发布").toString();
        jobInfo.setTime(time.substring(0, time.length() - 2));
        //把结果保存起来
        page.putField("jobInfo", jobInfo);
    }

    private Site site = Site.me()
            .setCharset("gbk")       //设置编码
            .setTimeOut(10 * 1000)   // 设置超时时间，单位毫秒
            .setRetrySleepTime(3000) // 设置重试的间隔时间
            .setRetryTimes(3);       // 设置重试次数

    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private SpringDataPipeline springDataPipeline;

    //initialDelay：当任务启动后，等待多久执行方法
    //fixedDelay：每隔多久执行一次方法
    @Scheduled(initialDelay = 1000, fixedDelay = 30 * 60 * 1000)
    public void process() {
        Spider.create(new JobProcessor())
                .addUrl(URL_51JOB)
                .setScheduler(new QueueScheduler().setDuplicateRemover(
                        new BloomFilterDuplicateRemover(1000000)))
                .thread(10)
                .addPipeline(this.springDataPipeline)
                .run();
    }

}
